import java.io.BufferedReader;
import java.io.IOException;
import java.io.InputStreamReader;
import java.io.Reader;
import java.net.MalformedURLException;
import java.net.URL;
import java.nio.charset.Charset;
import java.util.ArrayList;
import java.util.List;
import java.util.regex.Matcher;
import java.util.regex.Pattern;

public class WebSpiderTest
{

	public static String getURLSrcCode(String strUrl,String charSet)
	{
		StringBuilder sBuilder = new StringBuilder();
		URL url;
		try
		{
			url = new URL(strUrl);
			//以特定的格式打开输入流（不指定格式有乱码）
			BufferedReader reader = new BufferedReader(new InputStreamReader(url.openStream(),Charset.forName(charSet)));
			
			String temp = "";
			//从输入中读取字符串
			while((temp=reader.readLine())!=null)
			{
				//System.out.println(temp);
				sBuilder.append(temp+"\n");
			}
		}
		catch (MalformedURLException e)
		{
			// TODO Auto-generated catch block
			e.printStackTrace();
		}
		catch (IOException e)
		{
			// TODO Auto-generated catch block
			e.printStackTrace();
		}
		
		return sBuilder.toString();
		
	}
	
	/**
	 * 根据传入的规则和目标字符串，返回匹配结果
	 * @param desStr
	 * @param regStr
	 * @return
	 */
	public static List<String> getMatcherSubString(String desStr,String regStr)
	{
		Pattern pattern = Pattern.compile(regStr);
		Matcher matcher = pattern.matcher(desStr);
		List<String> result = new ArrayList<String>();
		
		while(matcher.find())
		{
//			System.out.println(matcher.group());
			System.out.println(matcher.group(1));
		}
		
		return result;
	}
	
	public static void main(String[] args)
	{
		String urlContet = getURLSrcCode("http://www.163.com","gbk");
		
		//超链接的这个地址 \s\S与.类似，但是比.多一个能匹配换行符
//		String patStr = "<a[\\s\\S]+?</a>";
		String patStr = "href=\"([\\w\\s./:]+?)\"";
		
		List<String> subStrList = getMatcherSubString(urlContet, patStr);
		
		for(int i=0;i<subStrList.size();i++)
		{
			System.out.println(subStrList.get(i));
		}
		
		
		
	}

}
